//==============================================================================================================================================================
// Author(s): Roman Khmelichek, Wei Jiang
//
//==============================================================================================================================================================

#include "external_index.h"

#include <cassert>
#include <cctype>
#include <cerrno>
#include <cstdlib>
#include <cstring>

#include <limits>

#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include "globals.h"
#include "logger.h"
#include "index_build.h"
using namespace std;
using namespace logger;

/**************************************************************************************************************************************************************
 * ExternalFrequencyData
 *
 **************************************************************************************************************************************************************/


/**************************************************************************************************************************************************************
 * ExternalDocumentData
 *
 **************************************************************************************************************************************************************/


/**************************************************************************************************************************************************************
 * ExternalChunk
 *
 **************************************************************************************************************************************************************/
ExternalChunk::ExternalChunk(){

}

ExternalChunk::~ExternalChunk(){
	// It proves that the following sentence doesn't help very much.
	// documents_data_.clear();
}

/**************************************************************************************************************************************************************
 * ExternalBlock
 *
 **************************************************************************************************************************************************************/
ExternalBlock::ExternalBlock() :
  block_max_score_(-numeric_limits<float>::max()) {
}

ExternalBlock::~ExternalBlock() {
}

/**************************************************************************************************************************************************************
 * ExternalIndexBuilder
 *
 *
 **************************************************************************************************************************************************************/
ExternalIndexBuilder::ExternalIndexBuilder(const char* external_index_filename) :
  external_index_fd_(open(external_index_filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)),
  curr_offset_(0) {
}

ExternalIndexBuilder::~ExternalIndexBuilder() {
  NewList();  // Flush anything we have left.
  close(external_index_fd_);
}

void ExternalIndexBuilder::AddChunk(const ChunkEncoder& chunk) {
  if (external_blocks_.empty())
  {
    NewBlock();
  }
  ExternalBlock& curr_external_block = external_blocks_.back();
  ExternalChunk external_chunk;
  external_chunk.set_chunk_max_score(chunk.max_score());
  curr_external_block.AddChunk(external_chunk);

}

void ExternalIndexBuilder::AddChunkWithScores(const ChunkEncoder& chunk) {
  if (external_blocks_.empty())
  {
    NewBlock();
  }
  ExternalBlock& curr_external_block = external_blocks_.back();
  ExternalChunk external_chunk;
  external_chunk.set_chunk_max_score( chunk.max_score() );

  //method 1: bad
  //external_chunk.set_stored_scores( chunk.scores_stored_in_index() );
  //external_chunk.set_number_of_stored_scores(chunk.number_of_scores_in_index());

  //method 2: good
  for(int i = 0; i < chunk.num_scores_in_index(); i ++){
	  ExternalDocumentData external_document_data;
	  external_document_data.set_partial_document_score_in_this_inverted_list(chunk.scores_in_index()[i]);
	  external_chunk.AddDocument(external_document_data);
  }


  curr_external_block.AddChunk(external_chunk);

}

// Start a new current block.
void ExternalIndexBuilder::NewBlock() {
  ExternalBlock new_external_block;
  external_blocks_.push_back(new_external_block);
}

// Used for both a new layer within a term or for a new term list.
// Means we also need to start a new block.
void ExternalIndexBuilder::NewList() {

  ssize_t write_ret;
  for (size_t i = 0; i < external_blocks_.size(); ++i) {
    const ExternalBlock& external_block = external_blocks_[i];
    int block_num_chunks = external_block.num_external_chunks();
    float block_max_score = external_block.block_max_score();


    write_ret = write(external_index_fd_, &block_num_chunks, sizeof(block_num_chunks));
    curr_offset_ += 1;
    assert(write_ret == sizeof(block_num_chunks));

    write_ret = write(external_index_fd_, &block_max_score, sizeof(block_max_score));
    curr_offset_ += 1;
    assert(write_ret == sizeof(block_max_score));

    for (int j = 0; j < block_num_chunks; ++j) {
      //ExternalChunk& external_chunk = external_block.external_chunk_num(j);
      // This is the original version of the define the variable external_chunk, const means that is only for read, but in fact, I have to clear the vector as well.
      const ExternalChunk& external_chunk = external_block.external_chunk_num(j);
      float chunk_max_score = external_chunk.chunk_max_score();

      write_ret = write(external_index_fd_, &chunk_max_score, sizeof(chunk_max_score));
      curr_offset_ += 1;
      assert(write_ret == sizeof(chunk_max_score));

      // Now, it is time to write the partial document score as well.
      int chunk_num_posting_scores = external_chunk.document_data().size();

      for(int k =0; k < chunk_num_posting_scores; ++k){
    	  float current_posting_score = external_chunk.document_data()[k].partial_document_score_in_this_inverted_list();
          write_ret = write(external_index_fd_, &current_posting_score, sizeof(current_posting_score));
          curr_offset_ += 1;
          assert(write_ret == sizeof(current_posting_score));
      }
    }
  }

  external_blocks_.clear();
}

/**************************************************************************************************************************************************************
 * ExternalIndexReader
 *
 **************************************************************************************************************************************************************/
ExternalIndexReader::ExternalIndexReader(const char* external_index_filename) :
  // original version by Roman.
  // external_index_fd_(open(external_index_filename, O_RDONLY)),

  // new version applied by Wei. Support read and write.
  external_index_fd_(open(external_index_filename, O_RDWR)),
  external_index_size_(0),
  external_index_(NULL) {
  struct stat stat_buf;
  if (fstat(external_index_fd_, &stat_buf) < 0) {
    GetErrorLogger().LogErrno("fstat() in ExternalIndexReader::ExternalIndexReader()", errno, true);
  }

  external_index_size_ = stat_buf.st_size;

  void* src;
  if ((src = mmap(0, external_index_size_, PROT_READ, MAP_SHARED, external_index_fd_, 0)) == MAP_FAILED) {
    GetErrorLogger().LogErrno("mmap() in MemoryMappedCachePolicy::MemoryMappedCachePolicy()", errno, true);
  }

  // Updated by Wei 2013/01/28
  // should be changed from uint32_t to uint64_t in order to support the big external index traversal
  // old version by Roman
  external_index_ = static_cast<uint32_t*> (src);

  // new version by Wei
  // external_index_ = static_cast<uint64_t*> (src);



  // Loop over the external index to make sure it's been read into memory.
  uint32_t external_index_data;
  assert(external_index_size_ % sizeof(*external_index_) == 0);
  uint64_t external_index_size_ints = external_index_size_ / sizeof(*external_index_);

  for (uint64_t i = 0; i < external_index_size_ints; ++i) {
    external_index_data = external_index_[i];
  }

  if(false){
	  cout << external_index_data << endl;
  }

  // for debug ONLY
  /*
  cout << "external_index_size_(in bytes):" << external_index_size_ << endl;
  cout << "external_index_:" << external_index_ << endl;
  cout << "sizeof(*external_index_):" << sizeof(*external_index_) << endl;
  cout << "sizeof(external_index_):" << sizeof(external_index_) << endl;
  cout << "external_index_size_(in ints):" << external_index_size_ints << endl;
  */
}

ExternalIndexReader::~ExternalIndexReader() {
  if (munmap(external_index_, external_index_size_) < 0) {
    GetErrorLogger().LogErrno("munmap() in ExternalIndexReader::~ExternalIndexReader()", errno, true);
  }
}

void ExternalIndexReader::AdvanceToBlock(uint32_t block_num, ExternalIndexPointer* external_index_pointer) const {
  // cout << "out 2.1.2.1" << endl;
  // cout << "ExternalIndexReader::AdvanceToBlock( " << block_num << ",...) called" << endl;
  // If we employed a in-memory block level index for block skipping, we have to properly skip the chunk scores.
  // And now, we have to properly skip the partial document score as well if we employed a in-memory document level index.
  int blocks_skipped = block_num - external_index_pointer->block_num_curr;

  //cout << "block_num:" << block_num << " external_index_pointer->block_num_curr:" << external_index_pointer->block_num_curr << " blocks_skipped:" << blocks_skipped << endl;

  assert(blocks_skipped >= 0);

  // cout << "out 2.1.2.2" << endl;
  // There might be some problem when dealing the block skipping, but Wei just don't care currently. By Wei:2012/07/03
  while (blocks_skipped) {
    cout << "BLOCK SKIPPING..." << endl;  //TODO: For debugging.

    DecodeBlock(external_index_pointer);
    // Use the variable external_index_pointer->offset_curr to control the operations of the whole external index.
    external_index_pointer->offset_curr += external_index_pointer->block_num_chunks;
    --blocks_skipped;
  }

  // cout << "out 2.1.2.3" << endl;
  while (external_index_pointer->block_num_curr <= block_num) {
    DecodeBlock(external_index_pointer);
  }

  // cout << "out 2.1.2.4" << endl;
  //This sentence is just used for getting rid of the c++ compiler's error.
  if(false){
	  cout << external_index_pointer->block_num_curr << endl;
  }
  // cout << "out 2.1.2.5" << endl;
}

void ExternalIndexReader::AdvanceToChunk(int chunk_num, ExternalIndexPointer* external_index_pointer) const {
	// cout << "ExternalIndexReader::AdvanceToChunk(" << chunk_num << ",...) called" << endl;

    // TODO: We store the actual number of chunks in the external index. This assertion is by Roman. Need to look at this problem.
    // assert(chunk_num < external_index_pointer->block_num_chunks);

	if (external_index_pointer->block_curr_chunk_num == -1) {
		external_index_pointer->block_curr_chunk_num = chunk_num;
		// TODO: Can store 'chunk_num' - 'external_index_pointer->block_num_chunks' + 1
		//       as the actual number of chunks in the block, and use that as the assertion above.
    }

  while (external_index_pointer->block_curr_chunk_num <= chunk_num) {
	DecodeChunk(external_index_pointer);
  }
}

void ExternalIndexReader::AdvanceToPosting(int posting_num, ExternalIndexPointer* external_index_pointer,bool debugFlag) const {
	if (external_index_pointer->chunk_curr_posting_num == -1) {
		if(debugFlag){
			//cout << "external_index_pointer->chunk_curr_posting_num == -1 and has been set to:" << posting_num << endl;
			cout << "external_index_pointer->chunk_curr_posting_num == -1 and has been set to: 0" << endl;
		}
		external_index_pointer->chunk_curr_posting_num = 0;
    }

	if(debugFlag){
		cout << "external_index_pointer->chunk_curr_posting_num:" << external_index_pointer->chunk_curr_posting_num << endl;
		cout << "posting_num:" << posting_num << endl;
	}

    while (external_index_pointer->chunk_curr_posting_num <= posting_num) {


    	DecodePosting(external_index_pointer);
    }
}

void ExternalIndexReader::AdvanceToPostingOpposite(int posting_num, ExternalIndexPointer* external_index_pointer,bool debugFlag, float new_score) const {
	if (external_index_pointer->chunk_curr_posting_num == -1) {
		if(debugFlag){
			//cout << "external_index_pointer->chunk_curr_posting_num == -1 and has been set to:" << posting_num << endl;
			cout << "external_index_pointer->chunk_curr_posting_num == -1 and has been set to: 0" << endl;
		}
		external_index_pointer->chunk_curr_posting_num = 0;
    }

	if(debugFlag){
		cout << "external_index_pointer->chunk_curr_posting_num:" << external_index_pointer->chunk_curr_posting_num << endl;
		cout << "posting_num:" << posting_num << endl;
	}

    while (external_index_pointer->chunk_curr_posting_num <= posting_num) {

    	if(external_index_pointer->chunk_curr_posting_num == posting_num){
    		DecodePostingOpposite(external_index_pointer, new_score, true);
    	}
    	else{
    		DecodePostingOpposite(external_index_pointer, new_score, false);
    	}

    }
}

// Decodes current block being pointed to by the 'external_index_pointer' and advances the pointer to the next block to be decoded.
void ExternalIndexReader::DecodeBlock(ExternalIndexPointer* external_index_pointer) const {
  // cout << "out 2.1.2.3.1" << endl;
  // cout << "ExternalIndexReader::DecodeBlock() called" << endl;
  // Updated by Wei 2013/01/28
  // (1) I don't know the purpose of the following assertion for
  // (2) I change the type from uint32_t to uint64_t, so affect the following assertion
  // (3) Maybe there is some problem
  assert(sizeof(external_index_pointer->offset_curr) == sizeof(external_index_pointer->block_num_chunks));

  external_index_pointer->block_curr_chunk_num = -1;

  // cout << "out 2.1.2.3.2" << endl;
  // cout << "external_index_:" << external_index_ << endl;
  // cout << "external_index_pointer->offset_curr:" << external_index_pointer->offset_curr << endl;
  // cout << "external_index_ + external_index_pointer->offset_curr:" << external_index_ + external_index_pointer->offset_curr << endl;

  memcpy(&external_index_pointer->block_num_chunks, external_index_ + external_index_pointer->offset_curr, sizeof(external_index_pointer->block_num_chunks));
//  external_index_pointer->block_num_chunks = external_index_[external_index_pointer->offset_curr];
  external_index_pointer->offset_curr += 1;

  // cout << "out 2.1.2.3.3" << endl;
  memcpy(&external_index_pointer->block_max_score, external_index_ + external_index_pointer->offset_curr, sizeof(external_index_pointer->block_max_score));
//  external_index_pointer->block_max_score = external_index_[external_index_pointer->offset_curr];
  external_index_pointer->offset_curr += 1;

  external_index_pointer->block_num_curr += 1;
  // cout << "out 2.1.2.3.4" << endl;
}

void ExternalIndexReader::DecodeChunk(ExternalIndexPointer* external_index_pointer) const {
  // cout << "ExternalIndexReader::DecodeChunk() called" << endl;

  // We always advance through every chunk in a block (though we don't decode every chunk in the primary index).
  external_index_pointer->chunk_curr_posting_num = -1;
  memcpy(&external_index_pointer->chunk_max_score, external_index_ + external_index_pointer->offset_curr, sizeof(external_index_pointer->chunk_max_score));
  external_index_pointer->offset_curr += 1;
  external_index_pointer->block_curr_chunk_num += 1;
}

void ExternalIndexReader::DecodePosting(ExternalIndexPointer* external_index_pointer) const {
  // We always advance through every posting in a chunk (though we may not decode every posting in the primary index).
  memcpy(&external_index_pointer->current_partial_document_score, external_index_ + external_index_pointer->offset_curr, sizeof(external_index_pointer->current_partial_document_score));
  external_index_pointer->offset_curr += 1;
  external_index_pointer->chunk_curr_posting_num += 1;
}

void ExternalIndexReader::DecodePostingOpposite(ExternalIndexPointer* external_index_pointer, float new_score, bool write_flag) const {
    // We always advance through every posting in a chunk (though we may not decode every posting in the primary index).

    // old version, may copy multiple times, but the final time is going to make sure be right.
    //memcpy(&external_index_pointer->current_partial_document_score, external_index_ + external_index_pointer->offset_curr, sizeof(external_index_pointer->current_partial_document_score));

	// new version, the following sentence may have a problem.
	if(write_flag){
		external_index_pointer->current_partial_document_score = new_score;
		//cout << "external_index_pointer->current_partial_document_score:" << external_index_pointer->current_partial_document_score << endl;
		//cout << "offset_curr in (4 bytes):" << external_index_pointer->offset_curr << endl;
		//cout << "offset_curr in bytes:" << external_index_pointer->offset_curr*4 << endl;
		lseek(external_index_fd_, external_index_pointer->offset_curr*4, SEEK_SET);
		//Let's actual write operation to disk here.
		ssize_t write_ret;
		write_ret = write(external_index_fd_, &new_score, sizeof(new_score));
		assert(write_ret == sizeof(new_score));
		//cout << "Change the score in external index...Done" << endl;
	}

	external_index_pointer->offset_curr += 1;
    external_index_pointer->chunk_curr_posting_num += 1;
}



/**************************************************************************************************************************************************************
 * ExternalIndexReader::ExternalIndexListData
 *
 **************************************************************************************************************************************************************/
ExternalIndexReader::ExternalIndexPointer::ExternalIndexPointer(uint32_t block_num, uint64_t offset) :
  offset_start(offset),
  offset_curr(offset_start),
  block_num_start(block_num),
  block_num_curr(block_num_start),
  block_curr_chunk_num(-1),
  chunk_curr_posting_num(-1),
  block_num_chunks(0),
  block_max_score(numeric_limits<float>::max()),
  chunk_max_score(numeric_limits<float>::max()),
  current_partial_document_score(numeric_limits<float>::max()){


  // cout << "initial_block_num:" << block_num << endl;
  // cout << "external_offset_start:" << offset_start << endl;
  // cout << "external_offset_curr:" << offset_curr << endl;
  // cout << "out1" << endl;

}

void ExternalIndexReader::ExternalIndexPointer::Reset() {
  offset_curr = offset_start;
  block_num_curr = block_num_start;
  block_num_chunks = 0;
  block_max_score = numeric_limits<float>::max();
  chunk_max_score = numeric_limits<float>::max();
}
